{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "Transformer_based_Classification_Spirit.ipynb",
   "provenance": [],
   "collapsed_sections": [
    "ygujDyznVK8y"
   ],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pwFyen9PamnP",
    "outputId": "03359f5e-3cb5-4a9d-9c2a-2905ee692a9e",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "!nvidia-smi"
   ],
   "execution_count": 1,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Tue Apr 27 03:52:55 2021       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
      "| N/A   38C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mLFhbl0yi63A",
    "outputId": "47bd3543-45c8-4f0d-de3b-9b43cb2f9d45",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "pip install -q tf-models-official"
   ],
   "execution_count": 4,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "\u001B[K     |████████████████████████████████| 1.1MB 12.6MB/s \n",
      "\u001B[K     |████████████████████████████████| 51kB 8.5MB/s \n",
      "\u001B[K     |████████████████████████████████| 706kB 32.3MB/s \n",
      "\u001B[K     |████████████████████████████████| 174kB 43.7MB/s \n",
      "\u001B[K     |████████████████████████████████| 102kB 12.9MB/s \n",
      "\u001B[K     |████████████████████████████████| 358kB 51.8MB/s \n",
      "\u001B[K     |████████████████████████████████| 645kB 55.7MB/s \n",
      "\u001B[K     |████████████████████████████████| 37.6MB 1.3MB/s \n",
      "\u001B[K     |████████████████████████████████| 1.2MB 53.2MB/s \n",
      "\u001B[?25h  Building wheel for seqeval (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "  Building wheel for py-cpuinfo (setup.py) ... \u001B[?25l\u001B[?25hdone\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Gx3lltd58zkg",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "import pickle"
   ],
   "execution_count": 5,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "cYQGTcl_86xy",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "import numpy as np\n",
    "from tensorflow.keras.utils import Sequence\n",
    "from tensorflow.keras.optimizers import SGD, Adam\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from official.nlp import optimization"
   ],
   "execution_count": 6,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "95b_uEPLgSIq",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ],
   "execution_count": 7,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "iqf3h3Sh88xN",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import classification_report"
   ],
   "execution_count": 8,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "W1kFJDLP8bwe",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers"
   ],
   "execution_count": 9,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Liu0qGrAVNxp",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# II. Transformer"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "-PUKB02N_QTF",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "def get_angles(pos, i, d_model):\n",
    "    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
    "    return pos * angle_rates\n",
    "\n",
    "def positional_encoding(position, d_model):\n",
    "    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
    "                            np.arange(d_model)[np.newaxis, :],\n",
    "                            d_model)\n",
    "\n",
    "    # apply sin to even indices in the array; 2i\n",
    "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "\n",
    "    # apply cos to odd indices in the array; 2i+1\n",
    "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "\n",
    "    pos_encoding = angle_rads[np.newaxis, ...]\n",
    "\n",
    "    return tf.cast(pos_encoding, dtype=tf.float32)"
   ],
   "execution_count": 45,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "7taKfNmh8e5B",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "class TransformerBlock(layers.Layer):\n",
    "    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n",
    "        super(TransformerBlock, self).__init__()\n",
    "        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n",
    "        self.ffn = keras.Sequential(\n",
    "            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n",
    "        )\n",
    "        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.dropout1 = layers.Dropout(rate)\n",
    "        self.dropout2 = layers.Dropout(rate)\n",
    " \n",
    "    def call(self, inputs, training):\n",
    "        attn_output = self.att(inputs, inputs)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(inputs + attn_output)\n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output, training=training)\n",
    "        return self.layernorm2(out1 + ffn_output)"
   ],
   "execution_count": 46,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "IM6iBfZP6U6C",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "class PositionEmbedding(layers.Layer):\n",
    "    def __init__(self, max_len, vocab_size, embed_dim):\n",
    "        super(PositionEmbedding, self).__init__()\n",
    "        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n",
    "        # self.pos_emb = layers.Embedding(input_dim=max_len, output_dim=embed_dim)\n",
    "        self.pos_encoding = positional_encoding(max_len,\n",
    "                                                embed_dim)\n",
    " \n",
    "    def call(self, x):\n",
    "        # x = self.token_emb(x)\n",
    "        seq_len = tf.shape(x)[1]\n",
    "        # print(maxlen)\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "        # positions = tf.range(start=0, limit=maxlen, delta=1)\n",
    "        # positions = self.pos_emb(positions)\n",
    "        # print(x.shape, positions.shape)\n",
    "        # x = self.token_emb(x)\n",
    "        return x"
   ],
   "execution_count": 47,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mziDCzBZ8g2q",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "embed_dim = 768  # Embedding size for each token\n",
    "num_heads = 12  # Number of attention heads\n",
    "ff_dim = 2048  # Hidden layer size in feed forward network inside transformer\n",
    "max_len = 75\n",
    "num_layers = 1"
   ],
   "execution_count": 57,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "qC3Duqe08qkZ",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "def transformer_classifer(input_size, loss_object, optimizer, dropout=0.1):\n",
    "    inputs = layers.Input(shape=(max_len, embed_dim))\n",
    "    transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\n",
    "    embedding_layer = PositionEmbedding(100, 2000, embed_dim)\n",
    "    # print(inputs.shape)\n",
    "    x = embedding_layer(inputs)\n",
    "    # print(x.shape)\n",
    "    x = transformer_block(x)\n",
    "    x = layers.GlobalAveragePooling1D()(x)\n",
    "    x = layers.Dropout(dropout)(x)\n",
    "    x = layers.Dense(32, activation=\"relu\")(x)\n",
    "    x = layers.Dropout(dropout)(x)\n",
    "    outputs = layers.Dense(2, activation=\"softmax\")(x)\n",
    "    model = keras.Model(inputs=inputs, outputs=outputs)\n",
    "    model.compile(loss=loss_object, metrics=['accuracy'],\n",
    "                  optimizer=optimizer)\n",
    "    return model"
   ],
   "execution_count": 58,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Abg-kaEbXYKM",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Training/Testing"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "J3y6Us-99Jyk",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "class BatchGenerator(Sequence):\n",
    " \n",
    "    def __init__(self, X, Y, batch_size):\n",
    "        self.X, self.Y = X, Y\n",
    "        self.batch_size = batch_size\n",
    " \n",
    "    def __len__(self):\n",
    "        return int(np.ceil(len(self.X) / float(self.batch_size)))\n",
    " \n",
    "    def __getitem__(self, idx):\n",
    "        # print(self.batch_size)\n",
    "        dummy = np.zeros(shape=(embed_dim,))\n",
    "        x = self.X[idx * self.batch_size:min((idx + 1) * self.batch_size, len(self.X))]\n",
    "        X = np.zeros((len(x), max_len, embed_dim))\n",
    "        Y = np.zeros((len(x), 2))\n",
    "        item_count = 0\n",
    "        for i in range(idx * self.batch_size, min((idx + 1) * self.batch_size, len(self.X))):\n",
    "            x = self.X[i]\n",
    "            if len(x) > max_len:\n",
    "                x = x[-max_len:]\n",
    "            x = np.pad(np.array(x), pad_width=((max_len - len(x), 0), (0, 0)), mode='constant',\n",
    "                       constant_values=0)\n",
    "            X[item_count] = np.reshape(x, [max_len, embed_dim])\n",
    "            Y[item_count] = self.Y[i]\n",
    "            item_count += 1\n",
    "        return X[:], Y[:, 0]"
   ],
   "execution_count": 59,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "2Hry6lv0psLS",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    def __init__(self, d_model, warmup_steps=4000):\n",
    "        super(CustomSchedule, self).__init__()\n",
    " \n",
    "        self.d_model = d_model\n",
    "        self.d_model = tf.cast(self.d_model, tf.float32)\n",
    " \n",
    "        self.warmup_steps = warmup_steps\n",
    " \n",
    "    def __call__(self, step):\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps ** -1.5)\n",
    " \n",
    "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
   ],
   "execution_count": 51,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "8XZXMT8-9TtR",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "def train_generator(training_generator, validate_generator, num_train_samples, num_val_samples, batch_size,\n",
    "                      epoch_num, model_name=None):\n",
    "  \n",
    "    # learning_rate = CustomSchedule(768)\n",
    " \n",
    "    # optim = tf.keras.optimizers.Adam(learning_rate)\n",
    "    \n",
    "    optim = Adam()\n",
    "    epochs = epoch_num\n",
    "    steps_per_epoch = num_train_samples \n",
    "    num_train_steps = steps_per_epoch * epochs\n",
    "    num_warmup_steps = int(0.1*num_train_steps)\n",
    " \n",
    "    init_lr = 3e-4\n",
    "    optimizer = optimization.create_optimizer(init_lr=init_lr,\n",
    "                                              num_train_steps=num_train_steps,\n",
    "                                              num_warmup_steps=num_warmup_steps,\n",
    "                                              optimizer_type='adamw')\n",
    "    \n",
    "    loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n",
    " \n",
    "    model = transformer_classifer(768, loss_object, optimizer)\n",
    " \n",
    "    # model.load_weights(\"hdfs_transformer.hdf5\")\n",
    " \n",
    "    print(model.summary())\n",
    " \n",
    "    # checkpoint\n",
    "    filepath = model_name\n",
    "    checkpoint = ModelCheckpoint(filepath,\n",
    "                                 monitor='val_accuracy',\n",
    "                                 verbose=1,\n",
    "                                 save_best_only=True,\n",
    "                                 mode='max',\n",
    "                                 save_weights_only=True)\n",
    "    early_stop = EarlyStopping(\n",
    "        monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto',\n",
    "        baseline=None, restore_best_weights=True\n",
    "    )\n",
    "    callbacks_list = [checkpoint, early_stop]\n",
    "    \n",
    "    # class_weight = {0: 245., 1: 1.}\n",
    " \n",
    "    model.fit_generator(generator=training_generator,\n",
    "                        steps_per_epoch=int(num_train_samples / batch_size),\n",
    "                        epochs=epoch_num,\n",
    "                        verbose=1,\n",
    "                        validation_data=validate_generator,\n",
    "                        validation_steps=int(num_val_samples / batch_size),\n",
    "                        workers=16,\n",
    "                        max_queue_size=32,\n",
    "                        callbacks=callbacks_list,\n",
    "                        shuffle=True\n",
    "                        # class_weight=class_weight\n",
    "                        )\n",
    "    return model"
   ],
   "execution_count": 52,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "N_7UPOMJ9HBQ",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "def train(X, Y, epoch_num, batch_size, tx, ty, model_file=None):\n",
    "    X, Y = shuffle(X, Y)\n",
    "    n_samples = len(X)\n",
    "    train_x, train_y = X[:int(n_samples * 90 / 100)], Y[:int(n_samples * 90 / 100)]\n",
    "    val_x, val_y = X[int(n_samples * 90 / 100):], Y[int(n_samples * 90 / 100):]\n",
    "    \n",
    "    training_generator, num_train_samples = BatchGenerator(train_x, train_y, batch_size), len(train_x)\n",
    "    validate_generator, num_val_samples = BatchGenerator(val_x, val_y, batch_size), len(val_x)\n",
    " \n",
    "    print(\"Number of training samples: {0} - Number of validating samples: {1}\".format(num_train_samples,\n",
    "                                                                                       num_val_samples))\n",
    " \n",
    "    model = train_generator(training_generator, validate_generator, num_train_samples, num_val_samples, batch_size,\n",
    "                              epoch_num, model_name=model_file)\n",
    "    test_model(model, tx, ty, batch_size)\n",
    " \n",
    " \n",
    "def test_model(model, x, y, batch_size):\n",
    "    x, y = shuffle(x, y)\n",
    "    x, y = x[: len(x) // batch_size * batch_size], y[: len(y) // batch_size * batch_size]\n",
    "    test_loader = BatchGenerator(x, y, batch_size)\n",
    "    prediction = model.predict_generator(test_loader, steps=(len(x) // batch_size), workers=16, max_queue_size=32,\n",
    "                                         verbose=1)\n",
    "    prediction = np.argmax(prediction, axis=1)\n",
    "    y = y[:len(prediction)]\n",
    "    report = classification_report(np.array(y), prediction)\n",
    "    print(report)"
   ],
   "execution_count": 53,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "0BKGP36V9A1i",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "from collections import Counter"
   ],
   "execution_count": 42,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "wFM5sqXJ8tHg",
    "outputId": "84305f3f-944c-4c80-eb12-6b897fedb6e2",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "with open(\"neural-train.pkl\", mode=\"rb\") as f:\n",
    "    (x_tr, y_tr) = pickle.load(f)\n",
    "x_tr, y_tr = shuffle(x_tr, y_tr)\n",
    "print(Counter(y_tr))\n",
    "with open(\"neural-test.pkl\", mode=\"rb\") as f:\n",
    "    (x_te, y_te) = pickle.load(f)\n",
    "print(Counter(y_te))\n",
    "print(\"Data loaded\")"
   ],
   "execution_count": 61,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "460048 460048\n",
      "Counter({0: 446559, 1: 13489})\n",
      "Counter({0: 111664, 1: 3349})\n",
      "Data loaded\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9qC3cZmR9F26",
    "outputId": "2a5b5c05-62e7-4240-8d1f-39ca2ae38ec9",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "train(x_tr, y_tr, 20, 64, x_te, y_te, \"hdfs_transformer.hdf5\")"
   ],
   "execution_count": 62,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Number of training samples: 414043 - Number of validating samples: 46005\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         [(None, 75, 768)]         0         \n",
      "_________________________________________________________________\n",
      "position_embedding_5 (Positi (None, 75, 768)           0         \n",
      "_________________________________________________________________\n",
      "transformer_block_5 (Transfo (None, 75, 768)           31491584  \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_5 ( (None, 768)               0         \n",
      "_________________________________________________________________\n",
      "dropout_22 (Dropout)         (None, 768)               0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 32)                24608     \n",
      "_________________________________________________________________\n",
      "dropout_23 (Dropout)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 2)                 66        \n",
      "=================================================================\n",
      "Total params: 31,516,258\n",
      "Trainable params: 31,516,258\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ],
     "name": "stdout"
    },
    {
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:1844: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
      "  warnings.warn('`Model.fit_generator` is deprecated and '\n"
     ],
     "name": "stderr"
    },
    {
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "6469/6469 [==============================] - 1067s 165ms/step - loss: 0.6323 - accuracy: 0.7571 - val_loss: 0.0356 - val_accuracy: 0.9920\n",
      "\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.99197, saving model to hdfs_transformer.hdf5\n",
      "Epoch 2/20\n",
      "6469/6469 [==============================] - 1072s 166ms/step - loss: 0.0341 - accuracy: 0.9918 - val_loss: 0.0161 - val_accuracy: 0.9968\n",
      "\n",
      "Epoch 00002: val_accuracy improved from 0.99197 to 0.99676, saving model to hdfs_transformer.hdf5\n",
      "Epoch 3/20\n",
      "6469/6469 [==============================] - 1066s 165ms/step - loss: 0.0159 - accuracy: 0.9968 - val_loss: 0.0117 - val_accuracy: 0.9983\n",
      "\n",
      "Epoch 00003: val_accuracy improved from 0.99676 to 0.99835, saving model to hdfs_transformer.hdf5\n",
      "Epoch 4/20\n",
      "6469/6469 [==============================] - 1066s 165ms/step - loss: 0.0093 - accuracy: 0.9982 - val_loss: 0.0052 - val_accuracy: 0.9988\n",
      "\n",
      "Epoch 00004: val_accuracy improved from 0.99835 to 0.99880, saving model to hdfs_transformer.hdf5\n",
      "Epoch 5/20\n",
      "6469/6469 [==============================] - 1064s 164ms/step - loss: 0.0065 - accuracy: 0.9988 - val_loss: 0.0049 - val_accuracy: 0.9992\n",
      "\n",
      "Epoch 00005: val_accuracy improved from 0.99880 to 0.99917, saving model to hdfs_transformer.hdf5\n",
      "Epoch 6/20\n",
      "6469/6469 [==============================] - 1064s 164ms/step - loss: 0.0049 - accuracy: 0.9991 - val_loss: 0.0038 - val_accuracy: 0.9993\n",
      "\n",
      "Epoch 00006: val_accuracy improved from 0.99917 to 0.99930, saving model to hdfs_transformer.hdf5\n",
      "Epoch 7/20\n",
      "6469/6469 [==============================] - 1061s 164ms/step - loss: 0.0048 - accuracy: 0.9991 - val_loss: 0.0033 - val_accuracy: 0.9993\n",
      "\n",
      "Epoch 00007: val_accuracy did not improve from 0.99930\n",
      "Epoch 8/20\n",
      "6469/6469 [==============================] - 1064s 164ms/step - loss: 0.0039 - accuracy: 0.9992 - val_loss: 0.0044 - val_accuracy: 0.9990\n",
      "\n",
      "Epoch 00008: val_accuracy did not improve from 0.99930\n",
      "Epoch 9/20\n",
      "6469/6469 [==============================] - 1064s 164ms/step - loss: 0.0039 - accuracy: 0.9993 - val_loss: 0.0055 - val_accuracy: 0.9991\n",
      "\n",
      "Epoch 00009: val_accuracy did not improve from 0.99930\n",
      "Epoch 10/20\n",
      "6469/6469 [==============================] - 1062s 164ms/step - loss: 0.0033 - accuracy: 0.9993 - val_loss: 0.0029 - val_accuracy: 0.9995\n",
      "\n",
      "Epoch 00010: val_accuracy improved from 0.99930 to 0.99948, saving model to hdfs_transformer.hdf5\n",
      "Epoch 11/20\n",
      "6469/6469 [==============================] - 1063s 164ms/step - loss: 0.0021 - accuracy: 0.9995 - val_loss: 0.0023 - val_accuracy: 0.9996\n",
      "\n",
      "Epoch 00011: val_accuracy improved from 0.99948 to 0.99959, saving model to hdfs_transformer.hdf5\n",
      "Epoch 12/20\n",
      "6469/6469 [==============================] - 1064s 164ms/step - loss: 0.0018 - accuracy: 0.9996 - val_loss: 0.0021 - val_accuracy: 0.9996\n",
      "\n",
      "Epoch 00012: val_accuracy improved from 0.99959 to 0.99961, saving model to hdfs_transformer.hdf5\n",
      "Epoch 13/20\n",
      "6469/6469 [==============================] - 1068s 165ms/step - loss: 0.0020 - accuracy: 0.9996 - val_loss: 0.0025 - val_accuracy: 0.9996\n",
      "\n",
      "Epoch 00013: val_accuracy did not improve from 0.99961\n",
      "Epoch 14/20\n",
      "6469/6469 [==============================] - 1069s 165ms/step - loss: 0.0015 - accuracy: 0.9996 - val_loss: 0.0024 - val_accuracy: 0.9995\n",
      "\n",
      "Epoch 00014: val_accuracy did not improve from 0.99961\n",
      "Epoch 15/20\n",
      "6469/6469 [==============================] - 1068s 165ms/step - loss: 0.0013 - accuracy: 0.9997 - val_loss: 0.0028 - val_accuracy: 0.9993\n",
      "\n",
      "Epoch 00015: val_accuracy did not improve from 0.99961\n",
      "Epoch 16/20\n",
      "6469/6469 [==============================] - 1066s 165ms/step - loss: 0.0014 - accuracy: 0.9997 - val_loss: 0.0021 - val_accuracy: 0.9996\n",
      "\n",
      "Epoch 00016: val_accuracy did not improve from 0.99961\n",
      "Epoch 17/20\n",
      "6469/6469 [==============================] - 1066s 165ms/step - loss: 0.0015 - accuracy: 0.9997 - val_loss: 0.0023 - val_accuracy: 0.9996\n",
      "\n",
      "Epoch 00017: val_accuracy did not improve from 0.99961\n"
     ],
     "name": "stdout"
    },
    {
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:1905: UserWarning: `Model.predict_generator` is deprecated and will be removed in a future version. Please use `Model.predict`, which supports generators.\n",
      "  warnings.warn('`Model.predict_generator` is deprecated and '\n"
     ],
     "name": "stderr"
    },
    {
     "output_type": "stream",
     "text": [
      "1797/1797 [==============================] - 132s 73ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    111659\n",
      "           1       1.00      0.99      0.99      3349\n",
      "\n",
      "    accuracy                           1.00    115008\n",
      "   macro avg       1.00      1.00      1.00    115008\n",
      "weighted avg       1.00      1.00      1.00    115008\n",
      "\n"
     ],
     "name": "stdout"
    }
   ]
  }
 ]
}